
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;








import java.util.*;
public class ExpandQuery {
StopWord stopword;
	public ExpandQuery(StopWord s)
	{
		stopword=s;
	}
	public Query apply(Query query, Presult relevantResultset,
			Presult nonrelevantResultset) 
	{
		Query newquery;
		Directory relevantIndex, nonrelevantIndex;

		
		relevantIndex = createIndex(relevantResultset);
		
		nonrelevantIndex = createIndex(nonrelevantResultset);
		
		Term[] newterms =
			selectTerms(query, relevantIndex, relevantResultset.getSize(),nonrelevantIndex,nonrelevantResultset.getSize());
		
		newquery = addterms(query, newterms, relevantIndex);
		
		try {
			relevantIndex.close();
			nonrelevantIndex.close();
		} catch (IOException e) {
			System.err.println("Failed to close directory: "+ e);
		}
		
	    return newquery;
	}
	
	public Directory createIndex(Presult rlist)
	{
		Directory directory;
		Analyzer analyzer = new StandardAnalyzer();
		
		try {
		    
		    directory = new RAMDirectory();
		    IndexWriter iwriter = new IndexWriter(directory, analyzer,
		    		IndexWriter.MaxFieldLength.UNLIMITED);
		    iwriter.setMaxFieldLength(25000);
		    
		    Iterator it = rlist.iterator();
		    while (it.hasNext()) {
		    	Document doc = new Document();
			    Result r = (Result)it.next();
		    	String text = r.title+" "+r.summary; 
			    doc.add(new Field("body", text, Field.Store.YES,Field.Index.ANALYZED));
			    iwriter.addDocument(doc);
		    }
		    iwriter.optimize();
		    iwriter.close();
		    return directory;

		} catch (IOException e) {
			System.err.println("Failed to create directory: "+ e);
			return null;
		}
	}
	
	public Term[] selectTerms(Query q, Directory rel, int rel_size, Directory non, int non_size)
	{
        Term[] returnterm = new Term[2];
	    
		TermVector[] relTermVector =
			compute(rel, rel_size);
		TermVector[] nonTermVector =
			compute(non, non_size);
	    
		TermVector[] newTermVector
			= FinalVectors(relTermVector, nonTermVector);
		
		Arrays.sort(newTermVector);
		int newtermnum = 0;
		boolean alreadyInQuery;
		List OldQueryTerms = q.getList();
		
		for (int i = 0; i < newTermVector.length; i++) {
			
			alreadyInQuery = false;
			

			for (int j=0;j<OldQueryTerms.size();j++) {
				
				String oldqueryterm = (String)OldQueryTerms.get(j);
				if (oldqueryterm.equals(newTermVector[i].term.text())) {
					alreadyInQuery = true;
					break;
				}
			}
			if (alreadyInQuery) {
				System.out.println("the new term \""+newTermVector[i].term.text()+"\" has alreay in old terms");
				continue;
			}
			if(!stopword.getwords().contains(newTermVector[i].term.text()))
			{
				if(!matching(newTermVector[i].term.text(),".com")&&!matching(newTermVector[i].term.text(),".edu")&&!matching(newTermVector[i].term.text(),".cn")&&!matching(newTermVector[i].term.text(),".org")&&!matching(newTermVector[i].term.text(),".net"))
				{
			System.out.println("Adding new term \"" + newTermVector[i].term + "\"");
			returnterm[newtermnum++] = newTermVector[i].term;
				}else{System.out.println("invalid term: "+newTermVector[i].term.text());}
			}else{System.out.println("found stop word: "+newTermVector[i].term.text());}
			if (newtermnum >= 2)
				break;
		}
		
		return returnterm;
	}
	
	public TermVector[] compute(Directory index, int size)
	{
		List TermVectorList = new ArrayList();
		try
		{
		IndexReader ireader = IndexReader.open(index);
		TermEnum terms = ireader.terms();
		System.out.println(" ----------- "+ terms+ " terms ------------");
		while(terms.next())
		{
			TermVector tv = new TermVector(terms.term(),0,ireader.docFreq(terms.term()),0);
			TermDocs tf = ireader.termDocs(terms.term()); 
			while (tf.next()) {
				tv.tf += tf.freq(); 
			}
			double wei = (double)tv.tf*Math.log(25000/tv.df)/size;
			tv.weight=wei;
			System.out.println("Term \""+terms.term().text()+"\" tf: "+tv.tf+" df: "+tv.df+" weight: "+tv.weight);
			TermVectorList.add(tv);
		}
		
		ireader.close();
		

		return (TermVector[])TermVectorList.toArray(new TermVector[TermVectorList.size()]);
		}catch (IOException e) {
			System.err.println("Failed to open Directory: "+ e);
			return null;
		}
	}
	public TermVector[] FinalVectors(TermVector[] rel, TermVector[] nonrel)
	{
		for(int i=0; i<rel.length;i++)
		{
			for(int j=0; j<nonrel.length;j++)
			{
				if(rel[i].term.equals(nonrel[j].term))
				{
					rel[i].weight-=nonrel[j].weight;
					System.out.println("recalculate the term \""+ rel[i].term +"\" weight, update to "+rel[i].weight);
				}
			}
		}
		return rel;
	}
	
	private Query addterms(Query oldquery, Term[] newterms, Directory RelevantIndex)
	{
		List allterms = new ArrayList();
		
		try{
		for(int i =0;i<2;i++)
		 allterms.add(newterms[i]);
		
		List oldquerylist = oldquery.getList();
		for(int i =0;i<oldquerylist.size();i++)
		{
			allterms.add(new Term("body", (String)oldquerylist.get(i)));
		}
		System.out.println("initial new terms order: "+ allterms.toString());
		
		Term[][] termsInOrder = getPosition((Term[])allterms.toArray(new Term[allterms.size()]), IndexReader.open(RelevantIndex));
		
		
		
		int[] calculatedPosi = new int[allterms.size()];
		
		int[] docWeight = new int[allterms.size()];
		for (int i = 0; i < termsInOrder.length; i++) {//every doc
			if (termsInOrder[i] == null)
				break;
			for(int j = 0; j< termsInOrder[i].length; j++){//every term in doc
				for (int k = 0; k < allterms.size(); k++) {//every term
					if(termsInOrder[i][j].equals((Term)allterms.get(k))){
						calculatedPosi[k]=calculatedPosi[k]+(j+1)*termsInOrder[i].length;
						docWeight[k]=docWeight[k]+termsInOrder[i].length;
						break;
					}
				}				
			}			

		}
		int isKey=0;
		for (int k = 0; k < allterms.size(); k++) {
			if(calculatedPosi[k]!=0){
				isKey++;
			}
		}

		double db;
		
		List posw=new ArrayList();
		
		for (int k = 0; k < allterms.size(); k++) {
			
			if(calculatedPosi[k]==0){
				calculatedPosi[k]=isKey+1;
				isKey++;
			}

			db=(double)calculatedPosi[k];
			if(docWeight[k]!=0){
				db = (double)calculatedPosi[k]/docWeight[k];
				System.out.println(allterms.get(k)+" "+(double)calculatedPosi[k]/docWeight[k]);
			}
			else
				System.out.println(allterms.get(k)+" "+calculatedPosi[k]);
			posw.add(new posweight(db,k));
			
			
		}
		posweight[] pw = (posweight[])posw.toArray(new posweight[posw.size()]);
		Arrays.sort(pw);
		Term[] t=new Term[allterms.size()];
		for (int k = 0; k < allterms.size(); k++) {
			t[k]= (Term) allterms.get(pw[k].getindex());
		}


		
		List finalBestTerms = new ArrayList();
		for (int i =0;i<t.length;i++) {
			
			finalBestTerms.add(t[i]);
		}
		
		
		System.out.println("final terms order: "+finalBestTerms.toString());
		return new Query((Term[])finalBestTerms.toArray(new Term[finalBestTerms.size()]), oldquery.iteration + 1);
		
	} catch (IOException e) {
		System.err.println(e);
		return null;
	}
	}
	
	private Term[][] getPosition(Term[] allOurTerms, IndexReader ireader) {
		Term[][] returnMe = new Term[10][];
		HashMap docToTerms = new HashMap();
		
		try {
			
			for (int i=0; i<allOurTerms.length;i++) {
				Term t = allOurTerms[i];
				TermPositions tp = ireader.termPositions(t);
				while (tp.next()) {
					int currDoc = tp.doc();
					Integer CurrDoc = new Integer(currDoc);
					int firstPos = tp.nextPosition(); 
					//System.out.println(" ************* t:"+t.text()+" doc:"+currDoc+" p:"+firstPos);
					if (!docToTerms.containsKey(CurrDoc))
						docToTerms.put(CurrDoc, new ArrayList());
					((ArrayList)docToTerms.get(CurrDoc)).add(new Position(t, currDoc, firstPos));
				}
			}
			
			Iterator iterator = docToTerms.keySet().iterator();
			int j = 0;
			while (iterator.hasNext()) {
				Integer key = (Integer)iterator.next();
				Term[] termsInOrderForADoc = new Term[((ArrayList)docToTerms.get(key)).size()];
				Position[] temp = (Position[])((ArrayList)docToTerms.get(key)).toArray(new Position[((ArrayList)docToTerms.get(key)).size()]);
				Arrays.sort(temp);
				for (int i = 0; i < temp.length; i++) {
					termsInOrderForADoc[i] = temp[i].term;
				}
				returnMe[j++] = termsInOrderForADoc;
			}
			
			return returnMe;
			
		} catch (IOException e) {
			System.err.println(e);
			return null;
		}
	}
	
	private boolean matching(String s1, String s2)
	{
		char[] c1=s1.toCharArray();
		char[] c2=s2.toCharArray();
		int j=0;
		int k=0;
		for(int i=0;i<c1.length;i++)
		{
			if(c1[i]==c2[j])
			{
				if(j==0)
					{
					if(c2.length==1)
						return true;
					else
					{
					k=i;j++;continue;
					}
					}
				j++;
				
				if(j==c2.length)
				{
					if(i-k+1==c2.length)
						return true;
				}
			}
		}
		return false;
	}
	private class posweight implements Comparable
	{
		double weight;
		int index;
		public posweight(double w, int i)
		{
			weight=w;
			index=i;
		}
		public double getweight()
		{return weight;}
		public int getindex()
		{return index;}
		
		public int compareTo(Object o)
		{
			if (this.weight < ((posweight)o).weight)
				return -1;
			else if (this.weight > ((posweight)o).weight)
				return 1;
			else
				return 0;
		}
		
	}
}
